import datetime
import json

import pandas as pd
import requests

import DBConst
import DateUtil
from mysql import SpiderTianyanchaHolderMSSQLDao
from mysql import MYSQLHelper
import SpiderTianyancha
import time
from Utils import JsonUtil
from Utils import MailUtil
from Utils import LoginUtil
from mysql import BaseDao
import StockConst
import urllib.request as request

# import sys
# import importlib
# importlib.reload(sys)
# sys.setsetdefaultencoding('utf8')

def getMysqlDatabaseDict():
    # 数据库
    M_IN_IP = 'ip'
    M_IN_ACCOUNT = 'account'
    M_IN_PASSWORD = 'password'
    M_IN_DATABASE = 'database'

    databaseDict = {M_IN_IP: DBConst.MY_IP,
                    M_IN_ACCOUNT: DBConst.MY_ACCOUNT,
                    M_IN_PASSWORD: DBConst.MY_PASSWORD,
                    M_IN_DATABASE: DBConst.MY_DATABASE_STOCK_INFO
                    }
    return databaseDict

# 全量
def processAllPage(lastUpdateDate, companyId):
    paramDict = {}
    for pageNum in range(1):
        # pageNum += 1
        url='http://www.tianyancha.com/expanse/holder.json?id='+str(companyId)+'&ps=20&pn=1'
        processOnePage(paramDict, url, pageNum, None, lastUpdateDate, companyId)
#
# # 更新前3页
# def processUpdate(lastUpdateDate, lastPageNum):
#     paramDict = {}
#     for pageNum in range(0, lastPageNum+1):
#         # pageNum += 1
#         url = 'http://ba.amac.org.cn/pages/amacWeb/user!list.action'
#         processOnePage(paramDict, url, pageNum, None, lastUpdateDate)

# 断点续传
# lastPage：上次最后一条的页码
# lastTableID：上次最后一条的ID
# def processContinue():
#
#     mysqlDatabaseDict = getMysqlDatabaseDict()
#     mysqlInst = MYSQLHelper.get_mssql_instance_by_dict(mysqlDatabaseDict)
#
#     ret = SpiderFundHomeMSSQLDao.selectLastRecord(mysqlInst)
#     lastTableID = ret[0]
#     lastPageNum = ret[1]
#     print(lastTableID)
#     print(lastPageNum)
#
#     paramDict = {}
#     for pageNum in range(lastPageNum, 1000):
#         # pageNum += 1
#         url = 'http://gs.amac.org.cn/amac-infodisc/api/pof/fund?rand=0.20962785827104136&page='+ str(pageNum)+'&size=20'
#         processOnePage(paramDict, url, pageNum, lastTableID)

isValidTodo = False

def processOnePage(paramDict, url, pageNum, lastTableID, lastUpdateDate, companyId):
    global isValidTodo
    # Cookie = "PHPStat_First_Time_10000011=1480428327337; PHPStat_Cookie_Global_User_Id=_ck16112922052713449617789740328; PHPStat_Return_Time_10000011=1480428327337; PHPStat_Main_Website_10000011=_ck16112922052713449617789740328%7C10000011%7C%7C%7C; VISITED_COMPANY_CODE=%5B%22600064%22%5D; VISITED_STOCK_CODE=%5B%22600064%22%5D; seecookie=%5B600064%5D%3A%u5357%u4EAC%u9AD8%u79D1; _trs_uv=ke6m_532_iw3ksw7h; VISITED_MENU=%5B%228451%22%2C%229055%22%2C%229062%22%2C%229729%22%2C%228528%22%5D"

    # # 数据库
    # M_IN_IP = 'ip'
    # M_IN_ACCOUNT = 'account'
    # M_IN_PASSWORD = 'password'
    # M_IN_DATABASE = 'database'

    databaseDict = getMysqlDatabaseDict()

    # headers = {
    #     'Accept': 'application/json, text/javascript, */*; q=0.01',
    #     'Accept-Encoding':'gzip, deflate',
    #     'Accept-Language': 'zh-CN,zh;q=0.8',
    #     'Connection':'keep-alive',
    #     'Content-Length': '2',
    #     'Content-Type': 'application/json',
    #     'Host':'gs.amac.org.cn',
    #     'Origin': 'http://gs.amac.org.cn',
    #     'Referer':'http://gs.amac.org.cn/amac-infodisc/res/pof/manager/managerList.html',
    #     'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36',
    #     'X-Requested-With':'XMLHttpRequest'
    # }

    # ===============================================================================================
    # 下面是3种方法：

    # # 方法1：这种方法无法取得动态内容
    # url = 'http://gs.amac.org.cn/amac-infodisc/res/pof/manager/managerList.html'
    # response = request.urlopen(url)
    # content = response.read().decode('utf-8')
    # print(content)

    # 方法2：加head的方式：报错
    # url = "http://gs.amac.org.cn/amac-infodisc/api/pof/manager?rand=0.3326844700932814&page=2&size=20"
    # req = request.Request(url, None, headers)
    # response = request.urlopen(req,timeout=5)
    # content = response.read().decode("utf-8")
    # print(content)

    # # 方法3：另存为后读本地html文件：可以
    # htmlfile = open('D:\\sourcehtml\\私募基金管理人分类公示 - 中国基金业协会.html', 'rb')  # 以只读的方式打开本地html文件
    # content = htmlfile.read().decode('utf-8')
    # print(content)

    # 获取动态网页

    # requestParam = {
    #     'filter_LIKES_MPI_NAME':'',
    #     'filter_LIKES_AOI_NAME':'',
    #     'filter_LIKES_MPI_PRODUCT_CODE':'',
    #     'filter_GES_MPI_CREATE_DATE':'',
    #     'filter_LES_MPI_CREATE_DATE':'',
    #     'page.searchFileName':'publicity_web',
    #     'page.sqlKey': 'PAGE_QH_PUBLICITY_WEB',
    #     'page.sqlCKey': 'SIZE_QH_PUBLICITY_WEB',
    #     '_search': 'false',
    #     'nd': '1495032782485',
    #     'page.pageSize': 50,
    #     'page.pageNo': 1,
    #     'page.orderBy': 'MPI_CREATE_DATE',
    #     'page.order': 'desc'
    # }

    # response = requests.post(url, None, {})
    # # response = requests.post(url,data = requestParam)
    # root = response.text
    # print(root)

    # domain = 'www.tianyancha.com'
    #
    # qcc_config = {'url': url, 'mid_str': domain}
    # content = LoginUtil.get_info_login(**qcc_config).text
    # print(content)

    response = request.urlopen(url)
    root = response.read().decode('utf-8')
    print(root)

    # check
    if root is None:
        return

    rootDict = json.loads(root)
    # print(companyDict)
    dataDict = rootDict['data']
    contentList = dataDict['result']
    # print(resultList)

    #
    #
    # #
    size = len(contentList)
    if size == 0:
        return

    date = datetime.datetime.now()
    resultList = []
    for i in range(size):
        print('home:'+str(i))
        # 第一行跳过
        companyDict = contentList[i]
        # id = companyDict['MPI_PRODUCT_CODE']

        # # 全量
        # if lastTableID is None:
        #     isValidTodo = True
        # # 断点续传
        # else:
        #     # 上次最后一条
        #     if lastTableID == id:
        #         isValidTodo = True
        #         continue
        #
        # # 非法
        # if not isValidTodo:
        #     continue

        # # 每日更新
        # mpiCreateDate = companyDict['MPI_CREATE_DATE']
        # if mpiCreateDate <= lastUpdateDate:
        #     continue

        companyDict['capitalActl_amomon'] = ''
        companyDict['capital_amomon'] = ''
        companyDict['capital_time'] = ''
        companyDict['capital_percent'] = ''
        companyDict['capital_paymet'] = ''

        if 'capitalActl' in companyDict.keys():
            capitalActl = companyDict['capitalActl']
            companyDict['capitalActl_amomon'] = capitalActl[0]['amomon']

        if 'capital' in companyDict.keys():
            capitalList = companyDict['capital']
            capitalDict = capitalList[0]
            if 'amomon' in capitalDict.keys():
                companyDict['capital_amomon'] = capitalDict['amomon']
            if 'time' in capitalDict.keys():
                companyDict['capital_time'] = capitalDict['time']
            if 'percent' in capitalDict.keys():
                companyDict['capital_percent'] = capitalDict['percent']
            if 'paymet' in capitalDict.keys():
                companyDict['capital_paymet'] = capitalDict['paymet']

        companyDict['CompanyUrlId'] = companyId
        companyDict['UpdateTime'] = DateUtil.datetime2_hhmmss_str(date)
        # companyDict['PageNum'] = str(pageNum)
        # companyDict['establishDate'] = DateUtil.timestampToDate(companyDict['establishDate'])
        # companyDict['putOnRecordDate'] = DateUtil.timestampToDate(companyDict['putOnRecordDate'])
        # resultList.append(companyDict)
        print(companyDict)
        SpiderTianyanchaHolderMSSQLDao.insert(databaseDict, companyDict)

        # SpiderAmacQHZC.insertedNum += 1

    # df = pd.DataFrame(resultList)
    # SpiderAmacQHZCMSSQLDao.insert(df)
    print('-------------------------------------SpiderFundHome end-------------------------------------')

insertedNum = 0
#
# # 更新任务
# def spiderUpdate():
#     # 1.计算开始结束时间
#     start_1 = time.clock()
#     currDate = DateUtil.get_current_time2()
#     lastUpdateDate = DateUtil.get_prev_date()  # currDate, -1
#     lastUpdateDateStr = DateUtil.datetime2_str(lastUpdateDate)
#     # lastPageNum = 2
#     # processUpdate(lastUpdateDateStr, lastPageNum)
#     processAllPage(StockConst.STOCK_BEGIN_DATE)
#     end_1 = time.clock()
#     diff = round(end_1 - start_1)
#     # 2.当前时间
#     currTime = DateUtil.get_current_time()
#     # 4.项目名，表名，插入数
#     projectName = 'SpiderTianyancha.py'
#     tableName = 'amac_QHZC'
#     # 3.前3条记录
#     mysqlDatabaseDict = getMysqlDatabaseDict()
#     mysqlInst = MYSQLHelper.get_mssql_instance_by_dict(mysqlDatabaseDict)
#     df = BaseDao.selectTop3(mysqlInst, currDate, tableName)
#     json_df = JsonUtil.df_json(df)
#     # 4.说明
#     remark = '期货资产管理计划'
#     # 5.发邮件
#     MailUtil.sendProjectEmail(currDate, currTime, diff, json_df, projectName, tableName, str(SpiderTianyancha.insertedNum), remark)
#
# # 每日任务
# def spiderDailyTask():
#     print('spiderDailyTask')
#     while True:
#         current_time = time.localtime(time.time())
#         h = current_time.tm_hour
#         m = current_time.tm_min
#         s = current_time.tm_sec
#         # 早上9点，清空SinaL2QuoteTemp
#         if ((h == 18) and (m == 0) and (s == 0)):
#             print("18:00:00 task")
#             spiderUpdate()
#             time.sleep(1)

if __name__ == "__main__":
    lastUpdateDate = StockConst.STOCK_BEGIN_DATE
    processAllPage(lastUpdateDate, 1698375)
    # processContinue()
    # spiderUpdate()
    # processUpdate('2017-05-12', 1)